package com.dddd.blog.blogsp.pageprocessor;

import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;


/**
 * @author 喵星球的大橘
 */
@Component
public class BlogAuthorPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

    private static final String BLOG_HOME = "https://www\\.cnblogs\\.com/\\w+/";

    private static final String AUTHOR_HOME = "https://home\\.cnblogs\\.com/u/\\w+/";

    private static final String URL_LIST = "https://www\\.cnblogs\\.com/pick/(\\d+/)?";

    private static final String URL_BASE = "https://www.cnblogs.com/pick/";

    private static Set<String> LIST_SET = new HashSet<>();

    private static String AUTHOR_INFO_SUFFIX = "ajax/news.aspx";

    private int page_start = 1;

    private int max_page = 20;


    @Override
    public void process(Page page) {

        Selectable url = page.getUrl();

        if(url.regex(URL_LIST).match()){
            //列表页
            if(page_start > max_page){
                return;
            }
            if(LIST_SET.contains(url.get())){
                return;
            }
            LIST_SET.add(url.get());

            String next = URL_BASE + (++page_start);

            List<String> all = page.getHtml().xpath("//*[@class='post-item-foot']/a[@class='post-item-author']").links().all();
            List<String> list = processList(all);
            page.addTargetRequests(list);
            page.addTargetRequest(next);
        }
        else if(url.regex(BLOG_HOME).match()){
            System.out.println("博客主页: "+url.get());
            System.out.println(url.get().split("/")[3]);

            page.putField("username",url.get().split("/")[3]);
            page.putField("homeLink",url.get());

        }else{
            return;
        }

    }

    private List<String> processList(List<String> list){
        List<String> newList = list.stream().distinct().collect(Collectors.toList());
        Iterator<String> it = newList.iterator();

        while(it.hasNext()){
            if(it.next().endsWith("/pick/")){
                it.remove();
            }
        }

        return newList;
    }


    @Override
    public Site getSite() {
        return site;
    }
}
